﻿using System;
using System.Data;
using System.Configuration;
using System.Collections;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
using Tong;
using System.Text.RegularExpressions;
public partial class aspnet_admin_bock_collect_CX_Collert_StratGetWebData : System.Web.UI.Page
{
    TongUse TextData = new TongUse();
    /// <summary>
    /// DataTable用于保存变量
    /// </summary>
    DataTable StartGetTable = new DataTable();
    /// <summary>
    /// 出错信息
    /// </summary>
    string ErrText ="";

    int addshu= 0;
    int Allshu = 0;
    /// <summary>
    /// 添加数据时的信息
    /// </summary>
    string WriteDataErr = "";
    protected void Page_Load(object sender, EventArgs e)
    {
        Button1.Attributes.Add("onclick", "loadinxxxx();");
        if (!IsPostBack)
        {
            DropDownList1.DataSource = TextData.SqlataSet("select * from [CX_Collect_Class] ");
            DropDownList1.DataTextField = "CX_Collect_Class_Name";
            DropDownList1.DataValueField = "CX_Collect_Class_id";
            DropDownList1.DataBind();

            DataTable CX_ReplacctTab = new DataTable();
            CX_ReplacctTab = TextData.SqlataSet("select * from [CX_Collect_Replace]").Tables[0];
            DataRow CX_ReplaceDataRow = CX_ReplacctTab.NewRow();
            CX_ReplaceDataRow[0] = "999999";
            CX_ReplaceDataRow[1] = "不进行替换";
            CX_ReplaceDataRow[2] = "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111";
            CX_ReplacctTab.Rows.Add(CX_ReplaceDataRow);
            Collect_Replace.DataSource = CX_ReplacctTab;
            Collect_Replace.DataTextField = "Cx_Collect_Relace_Name";
            Collect_Replace.DataValueField = "Cx_Collect_Relace_Contont";
            Collect_Replace.SelectedValue = "11111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111";
            Collect_Replace.DataBind();

            #region  关键字帮定操作 功能屏蔽 2007-10-11
            ////关键字帮定操作

            //功能屏蔽 2007-10-11

            //CX_ReplacctTab = TextData.SqlataSet("select Cx_keyWord_Name from [Cx_keyWord] ").Tables[0];
            ////DataRow CX_ReplaceDataRow1 = CX_ReplacctTab.NewRow();
            ////CX_ReplaceDataRow1[0] = "没有选择病种关键字"; //不进行加入关键字的值
            ////CX_ReplacctTab.Rows.Add(CX_ReplaceDataRow1);
            //Cx_keyWord_Name.DataSource = CX_ReplacctTab;
            //Cx_keyWord_Name.DataTextField = "Cx_keyWord_Name";
            //Cx_keyWord_Name.DataValueField = "Cx_keyWord_Name";
            ////Cx_keyWord_Name.SelectedValue = "没有选择病种关键字";
            //Cx_keyWord_Name.DataBind();
            #endregion

            CX_ReplacctTab = null;
            CX_ReplaceDataRow = null;
            
            
        }

        if (IsPostBack)
        {
            try
            {

                StartGetTable = TextData.SqlataSet("select * from [CX_Collect_Class] where CX_Collect_Class_id=" + DropDownList1.SelectedValue.Trim()).Tables[0];
                Label1.Text = "已选定:" + StartGetTable.Rows[0]["CX_Collect_Class_Name"].ToString();
                Cj_dz.Text = StartGetTable.Rows[0]["CX_Collect_Class_Page"].ToString();
                Cj_dz.NavigateUrl = StartGetTable.Rows[0]["CX_Collect_Class_Page"].ToString();
                Cj_dz.Target = "_blank";

                CJ_zz.Text = StartGetTable.Rows[0]["CX_Collect_Class_Creater"].ToString();
                CJ_CharSet.Text = StartGetTable.Rows[0]["CX_Collect_Class_CharSet"].ToString();


                CJ_Edit.NavigateUrl = "CX_Collert_Class_Edit.aspx?id=" + StartGetTable.Rows[0]["CX_Collect_Class_id"].ToString();
                CJ_Edit.Target = "_blank";
                StartGetTable.Dispose();//

            }
            catch
            {
                tt.Text = "出现未知错误";
            }
        }
    }

    //开始采集
    protected void Button1_Click(object sender, EventArgs e)
    {
        tt.Text = "";
        Labeltitle.Text = "";
        string[] imgsformWeb;//保存  图片本地路径的数组
        
        //得到首次的内容
        string ListRegStr = "";//得到列表的正择
        String ListCont = "";//得到列表的内容
        string GetUrlMain = "";//得到列表的页面
        string PageCode = "";
        string UpPathfies = "";//上传文件的路径

        //得到所有的A link 内容

        string   ListPageURL="";//得到要请求的页面列表
        string LinkRegStr = "";//得到所有A 的 href内容 的正规
        
        //开始得到内容页面的内容
        string[] ContPages;
        String Geturl = "";//得到内容的页面
        string ContRegStr = ""; //得到内容的正则
        string TietleRegstr = "";//得到标题的正则
        string PageContont = ""; //得到内容页面的内容

        /// 开始设定保存 到采集表的各个变量
        string CX_Collect_Class_Creater = "";//作者
        string CX_Collect_Name = "";//采集内容名
        string CX_Collect_Contont = "";//采集内容
        string CX_Collect_Class_Itemplitate = "";//
        string CX_Collect_Class_id = "";//采集分类设定
        string CX_Collect_FormURL = "";//采集来源
        string CX_Collect_Sitate = "0";//没有处理
        string CX_News_Class_id = "";//新闻分类ID
        string key_word = "";//病种关键字
        //添加数据的语句
        string sql = "";
        //如果没有新的采集地址

            DataTable CaiJiTab = new DataTable();
            try
            {
                CaiJiTab =   TextData.SqlataSet("select * from [CX_Collect_Class] where CX_Collect_Class_id=" + DropDownList1.SelectedValue.Trim()).Tables[0];
                ListRegStr = CaiJiTab.Rows[0]["CX_Collect_Class_List"].ToString();
                PageCode = CaiJiTab.Rows[0]["CX_Collect_Class_CharSet"].ToString();
                GetUrlMain = CaiJiTab.Rows[0]["CX_Collect_Class_Page"].ToString();
                LinkRegStr = CaiJiTab.Rows[0]["CX_Collect_Class_Link"].ToString(); //得到 LInk的正择
                ContRegStr = CaiJiTab.Rows[0]["CX_Collect_Class_Contont"].ToString(); //得到 内容
                TietleRegstr = CaiJiTab.Rows[0]["CX_Collect_Class_Title"].ToString(); //得到 标题
                CX_Collect_Class_Creater = CaiJiTab.Rows[0]["CX_Collect_Class_Creater"].ToString();//作者
                CX_Collect_Class_Itemplitate = CaiJiTab.Rows[0]["CX_Collect_Class_Itemplitate"].ToString();//mo板
                CX_Collect_Class_id = CaiJiTab.Rows[0]["CX_Collect_Class_id"].ToString();
                CX_News_Class_id = CaiJiTab.Rows[0]["CX_Column_id"].ToString(); //新闻分类ID
                UpPathfies = CaiJiTab.Rows[0]["CX_Collect_Class_UpFile"].ToString();//得到远程上传的文件路径

                //开始得到各标签 是不是不要
                bool TagA = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_A"].ToString());
                bool TagObject = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_OBJECT"].ToString());
                bool TagHTML = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_HTML"].ToString());
                bool TagSTYLE = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_STYLE"].ToString());
                bool TagDIV = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_DIV"].ToString());
                bool TagCLASS = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_CLASS"].ToString());
                bool TagFONT = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_FONT"].ToString());
                bool TagSPAN = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_SPAN"].ToString());
                bool TagIFRAME = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_IFRAME"].ToString());
                bool TagSCRIPT = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Filtrate_SCRIPT"].ToString());
        
                //得到看看是不是要远程保存图        
                bool IsGetWebPic = Convert.ToBoolean(CaiJiTab.Rows[0]["CX_Collect_Class_Parame"].ToString());

                if (urlx.Text.IndexOf("http://") != -1)
                {
                    GetUrlMain = urlx.Text;
                }

               

                ListCont = GetWebSite(GetUrlMain, PageCode);
                ListCont = Regex.Match(ListCont, RegexStrX(ListRegStr)).Value.ToLower();//把LINK列表页面的代码换成小写的

                if (ListCont != "")//如果有数据的话
                {

                    //

                //    TextBox1.Text = ListCont;

                    /////////////////////////以下代码开到得到 Alink
                    string getforntx;//得到前页面的东东
                    string getlastx;//得到正则后面的东东
                    string RegexStr = "";
                    MatchCollection ATag;///A标签
                    ATag = Regex.Matches(ListCont, @"<(\/){0,1}a[^<>]*>");

                    for (int i = 0; i < ATag.Count; i++)
                    {
                        
                        if (ATag[i].ToString().ToLower().IndexOf("href") != -1)
                        {
                            

                            //得到link
                            RegexStr = LinkRegStr.Replace("[链接到内容页]", "╊");
                            getforntx = RegexStr.Split('╊')[0];//得到前页面的东东
                            getlastx = RegexStr.Split('╊')[1];//得到正则后面的东东

                            //开始得到正则搜索到的数据 
                            RegexStr = Regex.Match(ATag[i].ToString(), RegexStrxContPage(LinkRegStr)).Value.ToString();
                            RegexStr = RegexStr.Replace(getforntx, "").Replace(getlastx, "");//这里以经可以 准确的输出 联接页面
                            RegexStr = TextData.FormAturl(RegexStr, GetUrlMain);
                            ListPageURL += RegexStr+"┵";

                        }

                    }
                    
                   // TextBox2.Text = ListPageURL;

                    if (ListPageURL.Replace("┵","") != "")
                    { 
                        ContPages = ListPageURL.Split('┵');//得到所有要请求的页面
                        Allshu = ContPages.Length;
                        try
                        {
                            if (Allshu <= int.Parse(Tiaoshu.Text.Trim()))//如果条数大于记录数
                            {
                                Allshu = ContPages.Length;
                            }
                            else
                            {
                                Allshu = int.Parse(Tiaoshu.Text.Trim());
                            }
                        }
                        catch
                        {
                            Tiaoshu.Text = "10";
                            tt.Text = "条数只能为数字 程序将10条执行";
                            if (Allshu <= 10)//如果条数大于记录数
                            {
                                Allshu = ContPages.Length;
                            }
                            else
                            {
                                Allshu = 10;
                            }

                        }
                        //得到所有数据
                        Labelcc.Text = "";
                        for (int j = 0; j <Allshu; j++)
                        {
                            Geturl = ContPages[j].ToString();

                            //开始得到数据
                            PageContont = GetWebSite(Geturl, PageCode);
                            if (PageContont != "")
                            {
                                CX_Collect_Name = Regex.Match(PageContont, RegexStrxContTitle(TietleRegstr)).Value.Replace("\r\n", "").Replace("\r", "").Replace("\n", ""); //这里是标题
                                CX_Collect_Name = Regex.Replace(CX_Collect_Name, @"<\/*[^<>]*>", "").Replace(" ", "").Replace("&nbsp;", "");//标题


                                CX_Collect_FormURL = Geturl;//得到采集来源
                                PageContont = Regex.Match(PageContont, RegexStrxCont(ContRegStr)).Value;//搜索到的内容



                                //// 如果有图片的话开始远程保有存图
                                //图片代码 <img src="file:///E|/承轩网络项目/NetDataBock/upfile/b764621a-bac6-453b-b531-b528695178b4.jpg" width="105" height="82" align="middle" /><br />
                                //开始执行图片保存操作
                                string Caji_imagespics = "";//用于保存所有远程图片生成后的HTML代码 "<img src=\"" + Cjpics + "\" border=\"0\"  align=\"middle\"/><br/>";
                                if (IsGetWebPic)
                                {

                                   

                                    try
                                    {
                                        imgsformWeb = TextData.GetWebImgSave(Geturl, PageContont, Tong.TongUse.SysPath + UpPathfies).Split('$');
                                        foreach (string Cjpics in imgsformWeb)
                                        {
                                            if (Cjpics != "")
                                            {
                                                Caji_imagespics += "<p align=center><img src=\"" + Cjpics + "\" border=\"0\"  align=\"middle\"/>\r\n</p><br/>";
                                            }
                                        }
                                    }
                                    catch
                                    {

                                    }
                                }



                                #region 开始定义标签 用于把文章里的东东全干掉

                                string ATagLink = @"<(\/){0,1}a[^<>]*>";///A标签
                                string ObjectTag = @"<object.*?/object>";///object 标签
                                string StyleTag = @"(<style){1,}[^<>]*>[^\0]*(<\/style>){1,}";/// style                            
                                string DivTag = @"<(\/){0,1}div[^<>]*>";//Div
                                string HTMLTag = @"<\/*[^<>]*>";
                                string CLASSTag = @"(class=){1,}(""|\'){0,1}\S+(""|\'|>|\s){0,1}";
                                string FONTTag = @"<(\/){0,1}font[^<>]*>";
                                string SPANTag = @"<(\/){0,1}span[^<>]*>";
                                string IFRAMETag = @"(<iframe){1,}[^<>]*>[^\0]*(<\/iframe>){1,}";
                                string SCRIPTTag = @"(<script){1,}[^<>]*>[^\0]*(<\/script>){1,}";

                                if (TagA)
                                {
                                    PageContont = Regex.Replace(PageContont, ATagLink, "");
                                }
                                if (TagObject)
                                {
                                    PageContont = Regex.Replace(PageContont, ObjectTag, "");
                                }

                                if (TagSTYLE)
                                {
                                    PageContont = Regex.Replace(PageContont, StyleTag, "");
                                }

                                if (TagDIV)
                                {
                                    PageContont = Regex.Replace(PageContont, DivTag, "");
                                }

                                if (TagCLASS)
                                {
                                    PageContont = Regex.Replace(PageContont, CLASSTag, "");
                                }
                                if (TagFONT)
                                {
                                    PageContont = Regex.Replace(PageContont, FONTTag, "");
                                }
                                if (TagSPAN)
                                {
                                    PageContont = Regex.Replace(PageContont, SPANTag, "");
                                }

                                if (TagIFRAME)
                                {
                                    PageContont = Regex.Replace(PageContont, IFRAMETag, "");
                                }

                                if (TagSCRIPT)
                                {
                                    PageContont = Regex.Replace(PageContont, SCRIPTTag, "");
                                }

                                if (TagHTML)
                                {
                                    PageContont = Regex.Replace(PageContont, HTMLTag, "");
                                }





                                #endregion
                                //把空格换成看得见的空格  换行也换了
                                CX_Collect_Name = Regex.Replace(CX_Collect_Name, HTMLTag, "");
                                CX_Collect_Name = CX_Collect_Name.Replace("（图）", "").Replace(" 图", "").Replace("图", "");//有的标题有图，要把所有的带图的全部干掉
                                CX_Collect_Name = CX_Collect_Name.Replace(Collect_Replace.SelectedValue, "");//替换所指定的关键字
                                PageContont = PageContont.Replace("\r\n", "").Replace(" \r\n", "<br>").Replace("   ", "&nbsp;").Replace("　 ", "&nbsp;&nbsp;").Replace("\t", "&nbsp;&nbsp;");

                                CX_Collect_Contont = PageContont + Caji_imagespics;//转换内容 +加入远程图片
                                CX_Collect_Contont = CX_Collect_Contont.Replace(Collect_Replace.SelectedValue, "");//替换所指定的关键字

                                #region  病种关键字功能屏蔽 2007-10-11
                                //if (Cx_keyWord_Name.SelectedValue.Trim() != "没有选择病种关键字")//病种关键字 已经选择的话
                                //{
                                //    key_word = Cx_keyWord_Name.SelectedValue.Trim().Replace("'","").Replace("\"","");
                                //}
                                #endregion

                                //开始执行添加数据
                                sql = "";
                                sql = "insert into [CX_Collect] ( ";
                                sql += "CX_Collect_Name,CX_Collect_Contont,CX_Collect_Class_Creater,CX_Collect_Class_Itemplitate,CX_Collect_Class_id,CX_Collect_FormURL,CX_Collect_Sitate,CX_NEWS_Class_id,Cx_keyWord_Name";
                                sql += ")";
                                sql += " values(";
                                sql += "'" + CX_Collect_Name.Replace("'", "") + "','" + CX_Collect_Contont.Replace("'", "") + "','" + CX_Collect_Class_Creater + "','" + CX_Collect_Class_Itemplitate + "'," + CX_Collect_Class_id + ",'" + CX_Collect_FormURL.Replace("'", "") + "'," + CX_Collect_Sitate + "," + CX_News_Class_id + ",'"+key_word+"'";
                                sql += ")";

                                ///看看是不是有同名标题 在同一栏目下
                                if (TextData.SqlataSet("select CX_Collect_Name from [CX_Collect] where CX_Collect_Name='" + CX_Collect_Name + "' and CX_Collect_Class_id=" + CX_Collect_Class_id).Tables[0].Rows.Count == 0 && CX_Collect_Name.Replace("'", "").Replace("\r", "").Replace("\n", "").Trim() != "" && CX_Collect_Contont.Replace("'", "").Replace("\r", "").Replace("\n", "").Trim() != "")
                                {
                                    TextData.SqlDataSave(sql);
                                    Labelcc.Text += "采集" + CX_Collect_Name + "成功" + "<br/>";
                                    addshu++;
                                }
                                else
                                {
                                    if (CX_Collect_Name.Replace("'", "").Replace("\r", "").Replace("\n", "").Trim() != "")
                                    {
                                        WriteDataErr += "采集标题为空,放弃写入" + "<br/>";
                                    }
                                    else
                                    {
                                        WriteDataErr += "写入" + CX_Collect_Name + "不成功存在同名标题 或标题内容为空" + "<br/>";
                                    }
                                }
                                

                                


                            }
                            
                        }
                        //Response.Write(sql);


                        Session["LabelTitle"] = CX_Collect_Name;
                        Session["LabelContont"]= CX_Collect_Contont;
                        Session["Labeltime"] = DateTime.Now.ToString();
                        Session["Labelfrom"] = CX_Collect_FormURL;
                        Session["LabelCreMan"] = CX_Collect_Class_Creater;
                        Labeltitle.Text = "<a href=\"CX_Collert_Viewtemp.aspx\" target=\"_blank\"> 查看  " + "</a>" + Session["LabelTitle"].ToString();
                        tt.Text ="应采集"+Allshu+"条 <br>已采集"+addshu+"条<br>出错"+(Allshu-addshu)+"条<br/>"+WriteDataErr;
                    }


                }
                else
                {
                    tt.Text = "没能从" + GetUrlMain + "通过采集条件得到数据 采集失败";
                }

            }
            catch (Exception ex)
            {
                tt.Text = "程序未能完成您的请求,由此给您带来的不便我们表示抱歉 <br/>请 查看:  采集设定   中是否能正确得到采集数据 ";
            }
 

    }

    /// <summary>
    /// 格式化得到列表内容的正 则
    /// </summary>
    /// <param name="Xstr">传入的字符</param>
    /// <returns></returns>
    public string RegexStrX(string Xstr)
    {
        return Xstr.Replace("[内容列表]", @"[\W\w]+");

    }


    /// <summary>
    /// 格式化得到列表内容的正 则
    /// </summary>
    /// <param name="Xstr">
    /// 传入正则
    /// </param>

    /// <returns></returns>
    public String RegexStrxContPage(string Xstr)
    {
        return Xstr.Replace("[链接到内容页]", @"[\r\n\s]{0,}.*");
    }


    /// <summary>
    /// HttpWebRequest读取网页 
    /// </summary>
    /// <param name="website">url</param>
    /// <returns></returns>
    protected string GetWebSite(string website,string pagecode)
    {
            string strHtmlContent = "";
        try
        {
        if (website.IndexOf("http://") == -1)//如果米有HTTP
        {
            website = "http://" + Request.Url.Authority + website;
        }
            System.Net.HttpWebRequest myrequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(website);
            myrequest.Timeout = 5000;
            // myrequest.Headers.Set("Pragma", "no-cache");
            System.Net.HttpWebResponse myresponse = (System.Net.HttpWebResponse)myrequest.GetResponse();
            System.IO.Stream mystream = myresponse.GetResponseStream();
            System.Text.Encoding myencoding = System.Text.Encoding.GetEncoding(pagecode);
            System.IO.StreamReader mystreamreader = new System.IO.StreamReader(mystream, myencoding);
            strHtmlContent = mystreamreader.ReadToEnd();
            mystream.Close();
            mystreamreader.Dispose();
            mystream.Close();
            mystream.Dispose();
            System.Threading.Thread.Sleep(1500);//1.5秒后执行下一条
           
        }
        catch
        {
             ErrText+="获得选程序文件出错 地址:"+website;
        }

        return strHtmlContent;
    }


    /// <summary>
    /// 格式化得到  页内容  的正 则
    /// </summary>
    /// <param name="Xstr">传入的字符</param>
    /// <returns></returns>
    public string RegexStrxCont(string Xstr)
    {
        return Xstr.Replace("[内容页内容]", @"[\W\w]+");

    }

    /// <summary>
    /// 格式化得到 页标题 
    /// </summary>
    /// <param name="Xstr">
    /// 传入正则
    /// </param>

    /// <returns></returns>
    public String RegexStrxContTitle(string Xstr)
    {
        return Xstr.Replace("[内容页标题]", @"[\W\w]+");
    }


}
